import java.io.*;
import java.util.Date;
import java.util.Hashtable;
import java.util.PriorityQueue;

public class TweetSplitter {

    static String regexp = "[\\s]+|[.]{2,}|,";
    // delimiters = white spaces as well as ellipses.
    // any occurrence of more than one period bunched together is taken as a delimiter.
    // single periods are taken care of later.
    // tweets tend to contain a lot of ellipses

    public static void main(String args[]) throws IOException {
        BufferedReader br;
        PrintStream writer;
        File directory = new File("/home/manigandan/Desktop/manispring2011/HW2/sample");
        File output = new File("/home/manigandan/Desktop/manispring2011/HW2/output");

        Hashtable<String, Word> words = new Hashtable<String, Word>();

        if (output.exists())
            if (!output.delete()) {
                System.err.println("Unable to cleanup output file! Exiting.");
                System.exit(-1);
            }

        if (!output.createNewFile()) {
            System.err.println("Unable to create new output file! Exiting.");
            System.exit(-1);
        }

        writer = new PrintStream(output);
        System.out.println("BEGIN : " + new Date(System.currentTimeMillis()));

        for (File file : directory.listFiles()) {
            br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));

            String str;
            while ((str = br.readLine()) != null) {
                str = str.substring(str.indexOf(" ") + 1).trim().toLowerCase();
                for (String s : str.split(regexp)) {
                    s = trimSpecialChars(s); // trims all special characters before and at the end of the string
                    s = trimRepetitions(s);
                    if (s.isEmpty()) continue;
                    Word w = words.get(s);
                    if (w == null)
                        words.put(s, new Word(s, 1));
                    else
                        w.increment();
                    //writer.println(s);
                }

            }
            br.close();

        }

//        System.out.println(trimSpecialChars("a---"));
//        System.out.println(trimSpecialChars("a-"));
//        System.out.println(trimSpecialChars("a"));
//        System.out.println(trimSpecialChars("-a-"));
//        System.out.println("--- : " + trimSpecialChars("---"));
//        System.out.println("- : " + trimSpecialChars("-"));


        System.out.println("VOCABULARY SIZE : " + words.keySet().size());

        PriorityQueue<Word> pQueue = new PriorityQueue<Word>();
        pQueue.addAll(words.values());
        System.out.println("END : " + new Date(System.currentTimeMillis()));


        PrintStream out = System.out;
        out = writer;
        for (int i = 1; i <= pQueue.size(); i++)
            out.println(pQueue.poll());

        System.out.println();
        writer.close();

    }

    static String trimRepetitions(String s) {
        if (s.length() <= 2) return s;
        StringBuilder result = new StringBuilder();
        result.append(s.charAt(0));
        boolean isDoubleRepetition = false;
        for (int i = 1; i <= s.length() - 1; i++) {
            if (s.charAt(i) != s.charAt(i - 1)) {
                if (isDoubleRepetition) result.append(s.charAt(i-1));
                result.append(s.charAt(i));
                isDoubleRepetition = false;
            }
            else {
                if (i >= 2 && s.charAt(i) == s.charAt(i-2))
                    isDoubleRepetition = false;
                else {
                    isDoubleRepetition = true;
                    if( i==s.length()-1)
                        result.append(s.charAt(i));
                }
            }
        }
        return result.toString();
    }

    static String trimSpecialChars(String str) {

        int startIndex = 0, endIndex = str.length() - 1;
        boolean start_isComplete = false, end_isComplete = false;


        while (true) {
            if (startIndex > endIndex) return "";
            if (!start_isComplete)
                if (Character.isLetterOrDigit(str.charAt(startIndex)))
                    start_isComplete = true;
                else
                    startIndex++;

            if (!end_isComplete)
                if (Character.isLetterOrDigit(str.charAt(endIndex)))
                    end_isComplete = true;
                else
                    endIndex--;

            if (startIndex > endIndex) return "";
            if (start_isComplete && end_isComplete) break;
        }

        return str.substring(startIndex, endIndex + 1);

    }
}

class Word implements Comparable<Word> {
    private String str;
    private int count;

    public Word(String s, int i) {
        str = s;
        count = i;
    }

    public String getString() {
        return str;
    }

    public String toString() {
        return str + " <" + count + ">";
    }

    public void increment() {
        count++;
    }

    public int compareTo(Word o) {
        return o.count - count;
    }

}
